From: kaf24@scramble.cl.cam.ac.uk Date: Thu, 15 Apr 2004 13:50:51 +0000 (+0000) Subject: bitkeeper revision 1.865 (407e933bgy9845pPJrw2YFAvCNIgEA) X-Git-Tag: archive/raspbian/4.8.0-1+rpi1~1^2~18256 X-Git-Url: https://dgit.raspbian.org/%22http:/www.example.com/cgi/%22https:/%22bookmarks://%22/%22http:/www.example.com/cgi/%22https:/%22bookmarks:/%22?a=commitdiff_plain;h=067e0972cb54923e66ba39754a9fcdf8359c9d00;p=xen.git bitkeeper revision 1.865 (407e933bgy9845pPJrw2YFAvCNIgEA) More TLB-flush cleanups. Simplify and rationalise the interface. --- diff --git a/xen/arch/i386/flushtlb.c b/xen/arch/i386/flushtlb.c index 9180454e4e..8712682d36 100644 --- a/xen/arch/i386/flushtlb.c +++ b/xen/arch/i386/flushtlb.c @@ -14,7 +14,7 @@ u32 tlbflush_clock; u32 tlbflush_time[NR_CPUS]; -static inline void tlb_clocktick(unsigned int cpu) +void tlb_clocktick(void) { u32 y, ny; @@ -34,23 +34,6 @@ static inline void tlb_clocktick(unsigned int cpu) } while ( unlikely((ny = cmpxchg(&tlbflush_clock, y-1, y)) != y-1) ); - /* Update cpu's timestamp to new time. */ - tlbflush_time[cpu] = y; + /* Update this CPU's timestamp to new time. */ + tlbflush_time[smp_processor_id()] = y; } - -void write_cr3_counted(unsigned long pa) -{ - __asm__ __volatile__ ( - "movl %0, %%cr3" - : : "r" (pa) : "memory" ); - tlb_clocktick(smp_processor_id()); -} - -void flush_tlb_counted(void) -{ - __asm__ __volatile__ ( - "movl %%cr3, %%eax; movl %%eax, %%cr3" - : : : "memory", "eax" ); - tlb_clocktick(smp_processor_id()); -} - diff --git a/xen/arch/i386/process.c b/xen/arch/i386/process.c index a9c2efd06d..ea5c51d176 100644 --- a/xen/arch/i386/process.c +++ b/xen/arch/i386/process.c @@ -280,8 +280,9 @@ void switch_to(struct task_struct *prev_p, struct task_struct *next_p) } } - /* Switch page tables. */ - write_ptbase( &next_p->mm ); + /* Switch page tables. */ + write_ptbase(&next_p->mm); + tlb_clocktick(); set_current(next_p); diff --git a/xen/arch/i386/setup.c b/xen/arch/i386/setup.c index 0eb4466291..c9eccfeb1d 100644 --- a/xen/arch/i386/setup.c +++ b/xen/arch/i386/setup.c @@ -282,8 +282,7 @@ void __init cpu_init(void) #undef CD /* Install correct page table. */ - __asm__ __volatile__ ("movl %%eax,%%cr3" - : : "a" (pagetable_val(current->mm.pagetable))); + write_ptbase(¤t->mm); init_idle_task(); } diff --git a/xen/common/domain.c b/xen/common/domain.c index 7eaa5628ad..053dfecef4 100644 --- a/xen/common/domain.c +++ b/xen/common/domain.c @@ -929,7 +929,7 @@ int construct_dom0(struct task_struct *p, /* Install the new page tables. */ __cli(); - write_cr3_counted(pagetable_val(p->mm.pagetable)); + write_ptbase(&p->mm); /* Copy the OS image. */ (void)loadelfimage(image_start); @@ -977,7 +977,7 @@ int construct_dom0(struct task_struct *p, *dst = '\0'; /* Reinstate the caller's page tables. */ - write_cr3_counted(pagetable_val(current->mm.pagetable)); + write_ptbase(¤t->mm); __sti(); /* Destroy low mappings - they were only for our convenience. */ diff --git a/xen/common/memory.c b/xen/common/memory.c index 4b13f84fb3..7c94748e07 100644 --- a/xen/common/memory.c +++ b/xen/common/memory.c @@ -871,6 +871,13 @@ static int do_extended_command(unsigned long ptr, unsigned long val) write_ptbase(¤t->mm); put_page_and_type(&frame_table[old_base_pfn]); + + /* + * Note that we tick the clock /after/ dropping the old base's + * reference count. If the page tables got freed then this will + * avoid unnecessary TLB flushes when the pages are reused. + */ + tlb_clocktick(); } else { @@ -1096,9 +1103,7 @@ int do_mmu_update(mmu_update_t *ureqs, int count) percpu_info[cpu].deferred_ops = 0; if ( deferred_ops & DOP_FLUSH_TLB ) - { - write_ptbase(¤t->mm); - } + local_flush_tlb(); if ( deferred_ops & DOP_RELOAD_LDT ) (void)map_ldt_shadow_page(0); @@ -1168,9 +1173,7 @@ int do_update_va_mapping(unsigned long page_nr, if ( unlikely(deferred_ops & DOP_FLUSH_TLB) || unlikely(flags & UVMF_FLUSH_TLB) ) - { - write_ptbase(&p->mm); - } + local_flush_tlb(); else if ( unlikely(flags & UVMF_INVLPG) ) __flush_tlb_one(page_nr << PAGE_SHIFT); diff --git a/xen/include/asm-i386/flushtlb.h b/xen/include/asm-i386/flushtlb.h index 4b558eae83..f0d4bb946c 100644 --- a/xen/include/asm-i386/flushtlb.h +++ b/xen/include/asm-i386/flushtlb.h @@ -43,9 +43,7 @@ static inline int NEED_FLUSH(u32 cpu_stamp, u32 lastuse_stamp) extern u32 tlbflush_clock; extern u32 tlbflush_time[NR_CPUS]; +extern void tlb_clocktick(void); extern void new_tlbflush_clock_period(void); -extern void write_cr3_counted(unsigned long pa); -extern void flush_tlb_counted(void); - #endif /* __FLUSHTLB_H__ */ diff --git a/xen/include/asm-i386/page.h b/xen/include/asm-i386/page.h index 54cbd85f11..61996d4ccc 100644 --- a/xen/include/asm-i386/page.h +++ b/xen/include/asm-i386/page.h @@ -98,7 +98,13 @@ typedef struct { unsigned long pt_lo; } pagetable_t; extern l2_pgentry_t idle_pg_table[ENTRIES_PER_L2_PAGETABLE]; extern void paging_init(void); -#define __flush_tlb() flush_tlb_counted() +#define __flush_tlb() \ + do { \ + __asm__ __volatile__ ( \ + "movl %%cr3, %%eax; movl %%eax, %%cr3" \ + : : : "memory", "eax" ); \ + tlb_clocktick(); \ + } while ( 0 ) /* Flush global pages as well. */ @@ -120,7 +126,7 @@ extern void paging_init(void); #define __flush_tlb_pge() \ do { \ __pge_off(); \ - flush_tlb_counted(); \ + __flush_tlb(); \ __pge_on(); \ } while (0) diff --git a/xen/include/asm-i386/processor.h b/xen/include/asm-i386/processor.h index cbce688e8b..26f64d1f9f 100644 --- a/xen/include/asm-i386/processor.h +++ b/xen/include/asm-i386/processor.h @@ -209,9 +209,6 @@ static inline unsigned int cpuid_edx(unsigned int op) #define X86_CR4_OSFXSR 0x0200 /* enable fast FPU save and restore */ #define X86_CR4_OSXMMEXCPT 0x0400 /* enable unmasked SSE exceptions */ -#define load_cr3(pgdir) \ - asm volatile("movl %0,%%cr3": :"r" (__pa(pgdir))); - /* * Save the cr4 feature set we're using (ie * Pentium 4MB enable and PPro Global page @@ -461,12 +458,16 @@ struct mm_struct { char gdt[6]; }; -static inline void write_ptbase( struct mm_struct *m ) +static inline void write_ptbase(struct mm_struct *mm) { - if ( unlikely(m->shadow_mode) ) - write_cr3_counted(pagetable_val(m->shadow_table)); + unsigned long pa; + + if ( unlikely(mm->shadow_mode) ) + pa = pagetable_val(mm->shadow_table); else - write_cr3_counted(pagetable_val(m->pagetable)); + pa = pagetable_val(mm->pagetable); + + __asm__ __volatile__ ( "movl %0, %%cr3" : : "r" (pa) : "memory" ); } #define IDLE0_MM \ diff --git a/xen/include/asm-x86_64/flushtlb.h b/xen/include/asm-x86_64/flushtlb.h index 4b558eae83..f0d4bb946c 100644 --- a/xen/include/asm-x86_64/flushtlb.h +++ b/xen/include/asm-x86_64/flushtlb.h @@ -43,9 +43,7 @@ static inline int NEED_FLUSH(u32 cpu_stamp, u32 lastuse_stamp) extern u32 tlbflush_clock; extern u32 tlbflush_time[NR_CPUS]; +extern void tlb_clocktick(void); extern void new_tlbflush_clock_period(void); -extern void write_cr3_counted(unsigned long pa); -extern void flush_tlb_counted(void); - #endif /* __FLUSHTLB_H__ */ diff --git a/xen/include/asm-x86_64/page.h b/xen/include/asm-x86_64/page.h index 0073a45c93..cb8651ec8a 100644 --- a/xen/include/asm-x86_64/page.h +++ b/xen/include/asm-x86_64/page.h @@ -162,7 +162,13 @@ extern unsigned long vm_force_exec32; extern l2_pgentry_t idle_pg_table[ENTRIES_PER_L2_PAGETABLE]; extern void paging_init(void); -#define __flush_tlb() flush_tlb_counted() +#define __flush_tlb() \ + do { \ + __asm__ __volatile__ ( \ + "movl %%cr3, %%eax; movl %%eax, %%cr3" \ + : : : "memory", "eax" ); \ + tlb_clocktick(); \ + } while ( 0 ) /* Flush global pages as well. */ @@ -184,7 +190,7 @@ extern void paging_init(void); #define __flush_tlb_pge() \ do { \ __pge_off(); \ - flush_tlb_counted(); \ + __flush_tlb(); \ __pge_on(); \ } while (0)